# coding=gb18030

import re
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import csv
import time
from database import DataBase as DB

class PoetrySpider:
    page_index=1
    title_index=1
    #url="http://www.shicimingju.com/chaxun/zuozhe/1.html"
    summary_css_selector='''body > div.layui-container
                            > div.layui-row.layui-col-space10
                            > div.layui-col-md4.layui-col-sm5
                            > div.zuozhe-header.www-shadow-card
                            > div.summary'''
    content_css_selector='''body > div.layui-container
                            > div.layui-row.layui-col-space10
                            > div.layui-col-md8.layui-col-sm7
                            > div.shici-container.www-shadow-card
                            > div.shici-content'''
    remark_css_selector='''body > div.layui-container
                           > div.layui-row.layui-col-space10
                           > div.layui-col-md8.layui-col-sm7
                           > div.shici-container.www-shadow-card
                           > div.shangxi-container'''
    title_css_selector_model='''body > div.layui-container
                                > div.layui-row.layui-col-space10
                                > div.layui-col-md8.layui-col-sm7
                                > div.www-shadow-card.www-main-container
                                > h3:nth-child({}) > a'''
    h3_css_selector='''body > div.layui-container 
                       >div.layui-row.layui-col-space10 
                       > div.layui-col-md8.layui-col-sm7 
                       > div.www-shadow-card.www-main-container 
                       > h3'''
    tag_css_selector='''body > div.layui-container 
                        > div.layui-row.layui-col-space10 
                        > div.layui-col-md8.layui-col-sm7 
                        > div.shici-container.www-shadow-card 
                        > div.shici-mark 
                        > a'''
    author_name_css_selector='''body > div.layui-container 
                                > div.layui-row.layui-col-space10 
                                > div.layui-col-md4.layui-col-sm5 
                                > div.zuozhe-header.www-shadow-card 
                                > h2 > a'''
    
    def __init__(self,url):
        
        self.chrome_options = Options()
        self.chrome_options.add_argument('--headless')
        self.browser=webdriver.Chrome(options=self.chrome_options)
        self.browser.get(url)
        self.author_name = self.browser.find_element_by_css_selector(self.author_name_css_selector).text
        self.db=DB()
        if self.db.is_table_exit("shici_1.db","poetries")==False:
            self.db.create_table("shici_1.db","poetries",["作者","标题","内容","注解","标签"],["TEXT","TEXT","TEXT","TEXT","TEXT"])
        else:
            pass
    def write_to_csv(self,rows,file_name=None):
        if file_name is None:
            file_name=self.csv_file
        self.pen=csv.writer(file_name)
        self.pen.writerow(rows)

    def get_poet_summary(self):
        try:
            summary=self.browser.find_element_by_css_selector(self.summary_css_selector).text
        except:
            summary=""
        db=DB()
        try:
            db.search_col("shici_1.db","authors","诗人详细介绍")
            pass
        except:
            db.add_new_col("shici_1.db","authors",("诗人详细介绍","TEXT"))
        db.update("shici_1.db","authors","诗人详细介绍",[summary,self.author_name])
        

    def from_page_get_h3_text_list(self):
        h3_obj_list=self.browser.find_elements_by_css_selector(self.h3_css_selector)
        h3_count=len(h3_obj_list)
        h3_text_list=[]
        for i in range(h3_count):
            h3_text_list.append(h3_obj_list[i].text)
        return h3_text_list

    def from_h3_list_get_title_list(self):
        h3_text_list=self.from_page_get_h3_text_list()
        title_list=[]
        p="《.*》"
        for h3_text in h3_text_list:
            title=re.search(p,h3_text).group(0)
            title_list.append(title)
        return title_list

    def goto_next_detail_page(self):
        self.browser.find_element_by_css_selector(self.title_css_selector_model.format(3*self.title_index)).click()

    def open_detail_page(self,poetry_title):
        self.browser.find_element_by_link_text(poetry_title).click()

    def back_to_last_page(self):
        self.browser.back()

    def get_one_poetry_details(self,poetry_title):
        self.goto_next_detail_page()
        poetry_content=self.browser.find_element_by_css_selector(self.content_css_selector).text
        if self.db.is_value_in_table("shici_1.db","poetries","内容","poetry_content")==True:
            pass
        else:
            try:
                poetry_remark=self.browser.find_element_by_css_selector(self.remark_css_selector).text
            except:
                poetry_remark=""
            tag_set=set()
            try:
                tag_obj_list=self.browser.find_elements_by_css_selector(self.tag_css_selector)
                for tag_obj in tag_obj_list:
                    tag_set.add(tag_obj.text)
            except:
                tag_set.add("")
            to_write_list=[self.author_name,poetry_title,poetry_content,poetry_remark,str(tag_set)[1:-1]]
            self.db.incert_data_list("shici_1.db","poetries",to_write_list)
            print("{}.{}:完成{}".format(self.page_index,self.title_index,poetry_title))
        self.title_index+=1
        self.back_to_last_page()

    def get_page_poetry_details(self):
        self.title_index=1
        title_list=self.from_h3_list_get_title_list()
        for title in title_list:
            self.get_one_poetry_details(title)

    def get_all_poetry_details(self):
        self.get_page_poetry_details()
        print("完成第{}页".format(self.page_index))
        try:
            self.browser.find_element_by_link_text("下一页").click()
            self.page_index+=1
            self.get_all_poetry_details()
        except:
            print("完成所有页")
            self.browser.close()


if __name__=="__main__":
    start=time.perf_counter()
    db_in_mian=DB()
    url_list=db_in_mian.search_col("shici_1.db","authors","诗人URL")
    index=1
    for url in url_list:
        if index <=4491:#这里的4491是已经完成抓取所有诗词的诗人的数量，可以使用SQL语句在数据库中查询得到。
            index+=1
            continue
        else:
            try:
                mySpider = PoetrySpider(url)
                mySpider.get_poet_summary()
                mySpider.get_all_poetry_details()
                finish=time.perf_counter()
                print("完成{}位作者的诗词抓取".format(index))
                print("共用时{:.2f}s".format(finish-start))
                index += 1
            except:
                print("Pass第{}位作者".format(index))
                index += 1
                continue
    print("done")